Breakdown of network to make sense of components.
- It was interesting to see the manually constructed loss function
- multi input network
- shared weights
Need to clean up the notebook a bit
In [1]:
'''Train a Siamese MLP on pairs of digits from the MNIST dataset.
It follows Hadsell-et-al.'06 [1] by computing the Euclidean distance on the
output of the shared network and by optimizing the contrastive loss (see paper
for mode details).
[1] "Dimensionality Reduction by Learning an Invariant Mapping"
http://yann.lecun.com/exdb/publis/pdf/hadsell-chopra-lecun-06.pdf
Gets to 99.5% test accuracy after 20 epochs.
3 seconds per epoch on a Titan X GPU
'''
Out[1]:
In [2]:
import numpy as np
import random
In [3]:
# Keras imports
from keras.datasets import mnist
from keras.models import Sequential, Model
from keras.layers import Dense, Dropout, Input, Lambda
from keras.optimizers import RMSprop
from keras import backend as K
In [4]:
def euclidean_distance(vects):
x, y = vects
return K.sqrt(K.maximum(K.sum(K.square(x - y), axis=1, keepdims=True), K.epsilon()))
In [5]:
def eucl_dist_output_shape(shapes):
shape1, shape2 = shapes
return (shape1[0], 1)
In [6]:
def contrastive_loss(y_true, y_pred):
'''Contrastive loss from Hadsell-et-al.'06
http://yann.lecun.com/exdb/publis/pdf/hadsell-chopra-lecun-06.pdf
'''
margin = 1
return K.mean(y_true * K.square(y_pred) +
(1 - y_true) * K.square(K.maximum(margin - y_pred, 0)))
In [7]:
def create_pairs(x, digit_indices):
'''Positive and negative pair creation.
Alternates between positive and negative pairs.
'''
pairs = []
labels = []
n = min([len(digit_indices[d]) for d in range(10)]) - 1
for d in range(10):
for i in range(n):
z1, z2 = digit_indices[d][i], digit_indices[d][i + 1]
pairs += [[x[z1], x[z2]]]
inc = random.randrange(1, 10)
dn = (d + inc) % 10
z1, z2 = digit_indices[d][i], digit_indices[dn][i]
pairs += [[x[z1], x[z2]]]
labels += [1, 0]
return np.array(pairs), np.array(labels)
In [8]:
def create_base_network(input_dim):
'''Base network to be shared (eq. to feature extraction).
'''
seq = Sequential()
seq.add(Dense(128, input_shape=(input_dim,), activation='relu'))
seq.add(Dropout(0.1))
seq.add(Dense(128, activation='relu'))
seq.add(Dropout(0.1))
seq.add(Dense(128, activation='relu'))
return seq
In [9]:
def compute_accuracy(predictions, labels):
'''Compute classification accuracy with a fixed threshold on distances.
'''
return labels[predictions.ravel() < 0.5].mean()
In [10]:
# the data, shuffled and split between train and test sets
(x_train, y_train), (x_test, y_test) = mnist.load_data()
In [11]:
x_train.shape, y_train.shape
Out[11]:
In [12]:
x_test.shape, y_test.shape
Out[12]:
In [13]:
x_train = x_train.reshape(60000, 784)
x_test = x_test.reshape(10000, 784)
x_train = x_train.astype('float32')
x_test = x_test.astype('float32')
So basically here we are just flattening the images dimensions
In [14]:
# normalize
x_train /= 255
x_test /= 255
# flatten images
input_dim = 784
epochs = 20
In [ ]:
In [ ]:
In [ ]:
In [15]:
# create training+test positive and negative pairs
digit_indices = [np.where(y_train == i)[0] for i in range(10)]
tr_pairs, tr_y = create_pairs(x_train, digit_indices)
digit_indices = [np.where(y_test == i)[0] for i in range(10)]
te_pairs, te_y = create_pairs(x_test, digit_indices)
In [67]:
tr_pairs.shape
Out[67]:
so this dimensions are basically:
- 108400 is the number of items in the list
- 2 each entry in the list has 2 parts (which are the pairs)
- 784 is the dimension of the flatten image
scratch that. There is no list anymore since the function calls for np.array() before returning
In [66]:
tr_pairs[:,0].shape
Out[66]:
In [68]:
tr_pairs[:,1].shape
Out[68]:
In [81]:
tr_y.shape
Out[81]:
In [17]:
len(digit_indices)
Out[17]:
In [23]:
np.where(y_test == 2)
Out[23]:
np.where returns a list, so you need the [0] to get a array only
In [24]:
np.where(y_test == 2)[0]
Out[24]:
so the code above creates an array for each of the numbers, each array has the indices for each of the numbers
create_pairs function
def create_pairs(x, digit_indices):
'''Positive and negative pair creation.
Alternates between positive and negative pairs.
'''
pairs = []
labels = []
n = min([len(digit_indices[d]) for d in range(10)]) - 1
for d in range(10):
for i in range(n):
z1, z2 = digit_indices[d][i], digit_indices[d][i + 1]
pairs += [[x[z1], x[z2]]]
inc = random.randrange(1, 10)
dn = (d + inc) % 10
z1, z2 = digit_indices[d][i], digit_indices[dn][i]
pairs += [[x[z1], x[z2]]]
labels += [1, 0]
return np.array(pairs), np.array(labels)
In [40]:
digit_indices[0][2], digit_indices[0][2+1]
Out[40]:
In [41]:
z1, z2 = digit_indices[0][2], digit_indices[0][2+1]
so we are getting the indices of two intances of the same number
In [43]:
x_test[z1][:10]
Out[43]:
In [44]:
x_test[z2][:10]
Out[44]:
then we are concatenating them
In [51]:
pairs += [[x_test[z1], x_test[z2]]]
In [52]:
len(pairs)
Out[52]:
so we keep building the pairs
In [53]:
inc = random.randrange(1, 10)
In [54]:
inc
Out[54]:
In [55]:
d = 4
In [56]:
dn = (d + inc) % 10
In [57]:
dn
Out[57]:
In [ ]:
# this adds the negative pair
# z1, z2 = digit_indices[d][i], digit_indices[dn][i]
# pairs += [[x[z1], x[z2]]]
the label is 1 for the first example and zero for the fake that we just made
In [ ]:
# labels += [1, 0]
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [26]:
pairs = []
labels = []
In [27]:
n = min([len(digit_indices[d]) for d in range(10)]) - 1
# how many indices are the per number
# get the min of all the indices
In [28]:
n
Out[28]:
In [29]:
for d in range(10):
print(len(digit_indices[d]))
In [30]:
# then it substract 1 - I assume its because the range is non-inclusive
In [35]:
n = 10
ran = 5
In [70]:
l = []
In [75]:
l += [1, 0]
In [76]:
l
Out[76]:
In [37]:
for d in range(ran):
for i in range(n):
z1, z2 = digit_indices[d][i], digit_indices[d][i + 1]
print(z1.shape, z2.shape)
pairs += [[x_train[z1], x_train[z2]]]
print(pairs)
inc = random.randrange(1, 10)
dn = (d + inc) % 10
z1, z2 = digit_indices[d][i], digit_indices[dn][i]
pairs += [[x_train[z1], x_train[z2]]]
labels += [1, 0]
np.array(pairs), np.array(labels)
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [59]:
# network definition
base_network = create_base_network(input_dim)
input_a = Input(shape=(input_dim,))
input_b = Input(shape=(input_dim,))
In [61]:
base_network.summary()
In [62]:
# because we re-use the same instance `base_network`,
# the weights of the network will be shared across the two branches
processed_a = base_network(input_a)
processed_b = base_network(input_b)
distance = Lambda(euclidean_distance,
output_shape=eucl_dist_output_shape)([processed_a, processed_b])
model = Model([input_a, input_b], distance)
In [63]:
model.summary()
In [ ]:
# loss is specified manually
# optimizer is RMSprop
In [77]:
tset = [tr_pairs[:, 0], tr_pairs[:, 1]]
In [79]:
len(tset)
Out[79]:
In [80]:
len(tr_y)
Out[80]:
In [28]:
# train
rms = RMSprop()
model.compile(loss=contrastive_loss, optimizer=rms)
model.fit([tr_pairs[:, 0], tr_pairs[:, 1]], tr_y,
batch_size=128,
epochs=epochs,
validation_data=([te_pairs[:, 0], te_pairs[:, 1]], te_y))
Out[28]:
In [14]:
# compute final accuracy on training and test sets
pred = model.predict([tr_pairs[:, 0], tr_pairs[:, 1]])
tr_acc = compute_accuracy(pred, tr_y)
pred = model.predict([te_pairs[:, 0], te_pairs[:, 1]])
te_acc = compute_accuracy(pred, te_y)
print('* Accuracy on training set: %0.2f%%' % (100 * tr_acc))
print('* Accuracy on test set: %0.2f%%' % (100 * te_acc))
In [ ]: